#include <zlib.h>

#include <signal.h>
#include <setjmp.h>

#include "LOGR_logs.h"
#include "LOGR_functions.h"
#include "LOGR_user_manager.h"
#include "LOGR_graph.h"

#define SIZEBUF 1024
#define SIZETOKEN 200

#define LOG_FILE_ERROR -1
#define LOG_DIRECTORY_ERROR -2

/*char *
  get_token_log(char **str, char delim);*/
void
print_counter_nodes(GraphFrame *gf);
int
increment_count_graph(GraphFrame *gf, char *rel_uri, char *referer,int size);

void
increment_count_edge(GraphFrame *gf, struct pt *v1, struct pt *v2);

extern void 
alrm_handler(int x);

int
LOGR_read_all_files_templates(GraphFrame *gf, char *sfilename, WebStatPtr wsn)
{
  
  char *ptr = strchr(sfilename,'*');
  if(!ptr)
    {
      if(LOGR_read_tokens(gf,sfilename,wsn) > 0)
	  return LOG_FILE_ERROR;  
    }
  else
    {
      DIR *dp;
      struct dirent *dirp;
      char *wodir = LOGR_parse_current_directory(sfilename);
      wodir[strlen(wodir)-1] = '\0';
      if((dp = opendir(wodir))==NULL)
	{
	  fprintf(stderr,"Cannot read directory %s\n",wodir);
	  return LOG_DIRECTORY_ERROR;
	}
      while((dirp = readdir(dp))!=NULL) 
	{
	  char *fname;
	  if(strcmp(dirp->d_name, ".") == 0 ||
	     strcmp(dirp->d_name, "..") == 0 )
	    continue;
	  fname = LOGR_complete_filename(sfilename, dirp->d_name);
	  if(fname && HTStrMatch(sfilename,fname))
	    {
	      printf("READING %s\n", fname);
	      if(LOGR_read_tokens(gf,fname,wsn) > 0)
		{
		  fprintf(stderr,"Log File %s is wrong\n",fname);
		  return LOG_FILE_ERROR;  
		}
	      printf("FINISH READING %s\n", fname);
	    }
	  if(fname) {
	      free(fname);  fname = NULL; }
	}
      if(closedir(dp)<0)
      {
	fprintf(stderr,"Cannot close directory %s\n", wodir);
	return LOG_DIRECTORY_ERROR;
      }
      free(wodir); wodir = NULL;
    }
  return 0;
}
void
print_counter_nodes(GraphFrame *gf)
{
  LNode_type *ptr, *lv = get_list_vertices(gf);

  for(ptr = lv->back  ; ptr->back !=  lv ; ptr = ptr->back)
    {
      struct pt *v = (struct pt * )ptr->info;
      if(v->counter)
	{
	  /*select_single_vertex(gf, v);*/
	  printf("%d: %d %s\n", v->num, v->counter, v->label);
	}
    }
}


void
flush_line(gzFile fp)
{
  int c;
  while((c = gzgetc(fp))!= EOF)
    {
      if(c == '\n')
	break;
    }
  c = gzgetc(fp);
}

gzFile
open_compressed_file(char *filename)
{
    struct stat statb;
    gzFile fp;

/* make sure the file is a regular text file and open it */
  if(stat(filename, &statb) == -1 ||
     (statb.st_mode & S_IFMT ) != S_IFREG ||
     !(fp = gzopen(filename, "r"))) 
    {
      if((statb.st_mode & S_IFMT) == S_IFREG)
	perror(filename);
      else
	fprintf(stderr, "%s : not a regular file \n", filename);
      return NULL;
    }
  return fp;
}

int
update_web_stats(GraphFrame *gf, WebStatPtr wsn, char *cursite, 
		 char *curdate, int http_code, int cur_size, 
		 char *referer,char *curbrowser,char *curl,int nhttp_method,
		 int nhttp_version)
{
    char *mime = curl ? HTGetMime(curl) : NULL;	
    int ishtmlpage = mime && (!strcmp(mime,"text/html") || !strcmp(mime,"www/unknown"));

    /*printf("CURL %s\n",curl);*/

    if(cursite) 
	LOGR_add_site(wsn,cursite,cur_size,ishtmlpage);

    if(curbrowser)
	LOGR_add_browser(wsn,curbrowser,cur_size,ishtmlpage);

    LOGR_add_http_code(wsn,http_code,cur_size,ishtmlpage);

    LOGR_add_http_method(wsn,(HttpMethodType)nhttp_method,
			 cur_size,ishtmlpage);
    LOGR_add_http_version(wsn,(HttpVersionType)nhttp_version,
			 cur_size,ishtmlpage);

    if(curdate)
	LOGR_add_date(wsn,curdate,ishtmlpage,cur_size);

    if(!curl)
	return 1;

    if(nhttp_method == HTTP_METHOD_GET && 
       (http_code == 200 || http_code >= 400))
    {
	TypeReferer tr = 0,trc = 0;
	TypeEdgeReferer ter = 0;
	int success = 0;

	trc = LOGR_type_url(curl);
	tr = LOGR_type_url(referer);
	ter = LOGR_type_edge_referer(wsn,referer);

	if(ter == INTERNAL_REFERER && tr == RSTATIC_REF)
	{
	    char *p = referer ? strchr(referer,'#') : NULL;
	    if(p)
		*p = '\0';
	}

	success = increment_count_graph(gf,curl,referer,cur_size);
	if(!success)
	{
	    /*printf("CURL %s\n",curl);*/
	    LOGR_add_independent_page(wsn,curl,cur_size,http_code,mime);
	}
	if(ter == EXTERNAL_REFERER)
	{
	    LOGR_add_entry_page(wsn, curl, gf,tr);
	    if(tr == RSTATIC_REF)
		LOGR_add_url_external_referer(wsn,referer,curl,cur_size,http_code);
	    else
	    {
		HTAssocList *al = HTParse_query(referer);
		LOGR_add_keywords(wsn, referer,al);
		HTAssocList_delete(al);
	    }
	}
	else if(ter == INTERNAL_REFERER)
	{
	    if(ishtmlpage)
		LOGR_add_url_internal_referer(wsn,referer,gf,cur_size);
	    if(trc == RSTATIC_REF)
		LOGR_add_independent_link(wsn, referer,curl,gf,curbrowser);
	    
	}
	else if(!referer)
	{
	    referer = "No referer";
	    ter = EXTERNAL_REFERER;
	    LOGR_add_entry_page(wsn, curl, gf,RNOREFERER);
	    LOGR_add_url_external_referer(wsn,referer,NULL,cur_size,http_code);
	}

	LOGR_user_manager(wsn, curdate, referer,curl, cursite,
			  curbrowser, ter, gf);

	if(referer)
	{
	    char * site_ref = HTParse(referer,"",PARSE_ACCESS|PARSE_HOST|PARSE_PUNCTUATION);
	    if(site_ref[0]!='\0')
		LOGR_add_site_ref3(wsn, site_ref,cur_size,tr);
	    else
		LOGR_add_site_ref3(wsn, "No Referer",cur_size,RNOREFERER);
	    free(site_ref); site_ref = NULL;
	}
	else
	    LOGR_add_site_ref3(wsn, "No Referer",cur_size,RNOREFERER);
	
    }

    return 0;
}

int
parse_log_line(char *bufline,int log_type, char **cursite,char **curdate,char **rel_uri,
               int *http_code,int *cur_size,char **referer,char **curbrowser,
               char **http_method, char **http_version)
{
    char *s = bufline;
    char *p = NULL;
    char *t = NULL;
    
    p = strchr(s,' ');
    if(!p) return -1;
    *p = '\0';
    p++;
    *cursite = s;
    /*printf("SITE %s\n", *cursite);*/
    
    s = strchr(p,'[');
    if(!s) return -1;
    s++;
    p = strchr(s,']');
    if(!p) return -1;;
    *p = '\0';
    p++;
    *curdate = s;
    /*printf("DATE %s\n", *curdate);*/
    
    s = strchr(p,'\"'); /* " */
    if(!s) return -1;
    s++;
    p = strchr(s,'\"'); /* " */
    if(!p) return -1;
    *p = '\0';
    p++;
    /*printf("HTTP %s\n", s);*/
    t = strchr(s,' ');
    if(!t) return -1;
    *t='\0';
    *http_method = s;
    /*printf("HTTP METHOD %s\n", *http_method);*/
    t++;
    s = strrchr(t,' ');
    if(!s) return -1;
    *s = '\0';
    *rel_uri = t;
    /*printf("REL URI %s\n", *rel_uri);*/

    s++;
    *http_version = s;
    /*printf("HTTP VERSION %s\n", *http_version);*/

    s = strchr(p,' ');    
    if(!s) return -1;
    s++;
    p = strchr(s,' ');
    if(!p) return -1;
    *p = '\0';
    p++;
    *http_code = atoi(s);
    /*printf("HTTP CODE %s <%d>\n", s,*http_code);*/

    s = p;
    p = strchr(s,' '); 

    if(!p) {
	if(log_type == LOGR_EXTENDED_LOG) {
	    fprintf(stderr,"Log File is not EXTENDED LOG File");	
	    return 1;
	}
	else if(log_type == LOGR_COMMON_LOG) {
	    *cur_size = atoi(s);
	    return 0;
	}
    }	
    else {
	if(log_type == LOGR_COMMON_LOG) {
	    fprintf(stderr,"Log File is not COMMON LOG File");	
	    return 2;
	}
    }
    *p = '\0';
    p++;
    *cur_size = atoi(s);
    /*printf("SIZE %s <%d>\n", s,*cur_size);*/

    s = strchr(p,'\"'); /* " */
    if(!s) return -1;
    s++;
    p = strchr(s,'\"'); /* " */
    if(!p) return -1;
    *p = '\0';
    p++;
    *referer = s;
    /*printf("REFERER %s\n",*referer);*/

    s = strchr(p,'\"'); /* " */
    if(!s) return -1;
    s++;
    p = strchr(s,'\"'); /* " */
    if(!p) return -1;
    *p = '\0';
    p++;
    *curbrowser = s;
    /*printf("USER AGENT %s\n",*curbrowser);*/

    return 0;
}

log_line *
process_log_line(char *bufline,WebStatPtr wsn,GraphFrame *gf,time_t *cdate,
		 int *fdate)
{
  int log_type = 0;
  const char *base_webhost = NULL;
  const char *url_prefix = NULL;
  int ferr = 0;

  char *cursite = NULL;
  char *curdate = NULL;	
  char *rel_uri = NULL;
  int http_code = 0;
  int cur_size = 0;
  char *referer = NULL;
  char *curbrowser = NULL;
  char *curl = NULL;
  char *areferer = NULL;
  char *http_version = NULL;
  char *http_method = NULL;
  int  nhttp_method = 0;
  int nhttp_version = 0;
  int delc = 0;
  int dela = 0;
  time_t tcdate;
  log_line *line = NULL;

  if(!wsn) return -1;

  log_type = LOGR_get_logs_filetype(wsn);
  base_webhost = LOGR_get_base_webhost(wsn);
  url_prefix = LOGR_get_url_prefix(wsn);

  ferr = parse_log_line(bufline, log_type, &cursite, &curdate, &rel_uri, 
			&http_code, &cur_size, &referer, &curbrowser,
			&http_method, &http_version);

  if(ferr) return ferr;

  nhttp_method = LOGR_get_http_method_type(http_method);
  nhttp_version = LOGR_get_http_version_type(http_version);

  if(*fdate == 0) {
      (*fdate)++;
      fprintf(stderr,"First date %s\n", curdate);
      tcdate = HTParseTimeApache(curdate);
      LOGR_set_start_date(wsn,tcdate);
  }

  if(curbrowser && curbrowser[0]=='-')
      curbrowser = NULL;

  if(rel_uri && base_webhost && !ferr)
  {
      /*curl = HTParse(HTUnEscape(rel_uri), base_webhost, PARSE_ALL);*/
      curl = LOGR_get_absolute_uri(wsn,gf,rel_uri,base_webhost,&delc);
      if(curl && url_prefix && !strstr(curl,url_prefix))
	  ferr = -1;
  }		

  if(referer && referer[0]!='-' && base_webhost && !ferr)
  {
      /*areferer = HTParse(HTUnEscape(referer), base_webhost, PARSE_ALL);
	areferer = HTSimplify(&areferer);*/
      areferer = LOGR_get_absolute_uri(wsn,gf,referer,base_webhost,&dela);
  }

  /*if(http_code >= 400)
      printf("Site %s date %s hc %d S %d R %s U %s UA %s HM %s HV %s\n", 
	 cursite ? cursite : "NULL",
	 curdate ? curdate : "NULL", http_code, cur_size,
	 areferer ? areferer : "NULL",
	 curl ? curl : "NULL",
	 curbrowser ? curbrowser : "NULL",
	 http_method ? http_method: "NULL",
	 http_version ? http_version: "NULL");*/
      
  if(!ferr) {

	update_web_graph(gf, cursite, curdate, http_code, cur_size, areferer,
		curbrowser, curl,http_method, nhttp_version);

    update_web_stats(gf, wsn, cursite, curdate, http_code, cur_size,areferer,
		     curbrowser,curl,nhttp_method,nhttp_version);

		if(!(line = (log_line *)malloc(sizeof(log_line)))) {
			perror("malloc()");
			abort();
		}

		line->cursite      = cursite;
		line->curdate      = curdate;
		line->httpCode     = http_code;
		line->bytes        = cur_size;
		line->referer      = areferer;
		line->browser      = curbrowser;
		line->curl         = curl;
		line->method       = nhttp_method;
		line->http_version = nhttp_version;
	}

  *cdate = HTParseTimeApache(curdate);

  if(curl && delc) {
      free(curl); curl = NULL;
  }

  if(areferer && dela) {
      free(areferer); areferer = NULL;
  }
  return line;
}
/*
int
process_log_line2(char *bufline,WebStatPtr wsn,GraphFrame *gf,char **cdate,
		 int *fdate)
{
  int log_type = 0;
  const char *base_webhost = NULL;
  const char *url_prefix = NULL;
  int i = 0;
  char *s = bufline;
  char *token = NULL;
  int ferr = 0;

  char *cursite = NULL;
  char *curdate = NULL;	
  char *rel_uri = NULL;
  int http_code = 0;
  int cur_size = 0;
  char *referer = NULL;
  char *curbrowser = NULL;	
  char *curl = NULL;
  char *areferer = NULL;

  if(!wsn) return -1;

  log_type = LOGR_get_logs_filetype(wsn);
  base_webhost = LOGR_get_base_webhost(wsn);
  url_prefix = LOGR_get_url_prefix(wsn);
  
  while(*s != '\0')
  {
    i++;

    token = get_i_token(&s,i);

    while(*s == ' ')
	s++;

    if(!token)
	    break;

   switch(i)
    {
    case 1:
	cursite = token;
	token = NULL;
	break;
    case 2:
	break;
    case 3:
	break;
    case 4:
	curdate = token;
	token = NULL;
	if(*fdate == 0) {
	    (*fdate)++;
	    fprintf(stderr,"First date %s\n", curdate);
	    LOGR_set_start_date(wsn,curdate);
	}
	break;
    case 5:
	rel_uri = NULL;
	if(!strncmp(token,"GET",3))
	    rel_uri = strtok(&(token[4])," ");
	if(rel_uri && base_webhost && !ferr)
	{
	    curl = HTParse(HTUnEscape(rel_uri), base_webhost, PARSE_ALL);
	    if(curl && url_prefix && !strstr(curl,url_prefix))
		ferr = -1;
	}
	break;
    case 6:
	http_code = atoi(token);
	break;
    case 7:
	cur_size = atoi(token);
	break;
    case 8:
	if(log_type == LOGR_EXTENDED_LOG)
	{	
	    if(token[0]!='-') 
	    {
		referer = token;
		token = NULL;
	    } 
	}
	else if(log_type == LOGR_COMMON_LOG)
	{
	   fprintf(stderr,"Log File is not COMMON LOG File");
	   ferr = 1; 
	}
	break;
    case 9:
	if(log_type == LOGR_EXTENDED_LOG)
	{	
	    if(token[0]!='-') 
	    {
		curbrowser = token;
		token = NULL;
	    }
	}
	else if(log_type == LOGR_COMMON_LOG)
	{
	   fprintf(stderr,"Log File is not COMMON LOG File");
	   ferr = 1; 
	}
	break;
    default:
	fprintf(stderr,"Log File is not EXTENDED LOG File");
	ferr = 2;
	break;
	} 

    if(token) {
	free(token); token = NULL;
    }

    if(ferr)
	break;
  }


  if(referer && base_webhost && !ferr)
  {
      areferer = HTParse(HTUnEscape(referer), base_webhost, PARSE_ALL);
      areferer = HTSimplify(&areferer);
  }

	 
  if(!ferr && curl)
    update_web_stats(gf, wsn, cursite, curdate, http_code, cur_size, areferer,
		     curbrowser,curl);

  if(cursite) {
      free(cursite); cursite = NULL;
  }

  *cdate = curdate;
  if(referer) {
      free(referer); referer = NULL;
  }
  if(areferer) {
      free(areferer); areferer = NULL;
  }
  if(curbrowser) {
      free(curbrowser); curbrowser = NULL;
  }
  if(curl) {
      free(curl); curl = NULL;
  }
  return ferr;
}
*/

int
LOGR_read_tokens(GraphFrame *gf, char *filename, WebStatPtr wsn)
{
  gzFile fp;
  int ferr = 0;
  long lineno = 0;
  char bufline[SIZEBUF];
  int log_type = LOGR_get_logs_filetype(wsn);
  time_t edate = 0;
  time_t ldate = 0;
  int fdate = 0;
  char pdate[100];
  static short verify = 0;

  if(!wsn) return -1;

  fp = open_compressed_file(filename);
  if(!fp)
      return -1;

  while(gzgets(fp, bufline,SIZEBUF)!= NULL && ferr <= 0)
  {
      lineno++;
      if(lineno%10000 == 0)
      fprintf(stderr,"%ld\n",lineno/10000);

      if(bufline)
      {
	  int len = strlen(bufline);
	  if(len >= SIZEBUF - 1)
	  {
	      fprintf(stderr, "LOG LINE IS TOO BIG\n");
	      flush_line(fp);
	      continue;
	  }
      }
      if(edate)
	  ldate = edate;
      ferr = process_log_line(bufline,wsn,gf,&edate,&fdate);
  }

  LOGR_process_browsers_type_version_platform(wsn);
  LOGR_process_directories(wsn,gf);
  LOGR_process_simple_pages(wsn,gf,verify);
  LOGR_process_simple_links(wsn,gf,verify);

  if(LOGR_get_resolve_host_ip(wsn)) 
  {
      signal(SIGALRM,alrm_handler);
      LOGR_process_sites(wsn,1);
  }

  LOGR_process_domains(wsn);

  if(ferr == 0 && log_type == LOGR_EXTENDED_LOG)
      fprintf(stderr,"\nFinish reading Extended Log File\n");
  if(ferr == 0 && log_type == LOGR_COMMON_LOG)
      fprintf(stderr,"\nFinish reading Common Log File\n");

  if(ldate > 0) {
      HTBufferTimeApache(ldate,pdate,100);
      printf("End date %s\n", pdate);
      LOGR_set_end_date(wsn,ldate);      
  } else if(edate) {
      HTBufferTimeApache(edate,pdate,100);
      printf("End date %s\n", pdate);
      LOGR_set_end_date(wsn,edate);      
  }

  gzclose(fp);
  verify = 1;
  return ferr;
}

int
increment_count_graph(GraphFrame *gf, char *auri, char *aref,int size)
{
  int success = 0;
  PNODE v1 = NULL, v2 = NULL;
  char bsize[20];
  
  if(!auri)
      return 0;

  v2 = find_vertex_by_name(gf, auri);
  if(v2) 
  {
      char *ssize = v2 ? (char *)get_attribute(v2,"size") : NULL;
      increment_counter_vertex(v2);
      if(!ssize)
      {
	  sprintf(bsize,"%d",size);
	  add_copy_attribute(v2,"size",bsize);
      }
      success = 1;
  }
  v1 = aref ? find_vertex_by_name(gf, aref) : NULL;
  if(v1 && v2) increment_count_edge(gf,v1,v2);

  return success;
}

void
increment_count_edge(GraphFrame *gf, struct pt *v1, struct pt *v2)
{
  struct edge *e = NULL;

  e = find_edge(gf, v1, v2);
  if(e)
    {
      char *eweight = get_weight_edge(e);
      char neww[10];
      int iew = atoi(eweight);
      iew++;
      sprintf(neww,"%d",iew);
      change_weight_edge(e,neww);      
    }
}

void
LOGR_start_application(void)
{
  HTMime_init();
}

void
LOGR_end_application(void)
{
  HTMime_deleteAll();
}

